Decorrelation-Based Feature Discovery: TADPOLE-Whole

1 Effect of UPSTM-Based Decorrelation on Feature Discovery

1.0.1 Loading the libraries

library("FRESA.CAD")
library(readxl)
library(igraph)
library(umap)
library(tsne)
library(entropy)

op <- par(no.readonly = TRUE)
pander::panderOptions('digits', 3)
pander::panderOptions('table.split.table', 400)
pander::panderOptions('keep.trailing.zeros',TRUE)

1.1 Material and Methods

Data from the speech features

1.2 The data set

TADPOLE_D1_D2 <- read.csv("~/GitHub/BSWiMS/Data/TADPOLE/TADPOLE_D1_D2.csv")
TADPOLE_D1_D2_Dict <- read.csv("~/GitHub/BSWiMS/Data/TADPOLE/TADPOLE_D1_D2_Dict.csv")
TADPOLE_D1_D2_Dict_LR <- as.data.frame(read_excel("~/GitHub/BSWiMS/Data/TADPOLE/TADPOLE_D1_D2_Dict_LR.xlsx",sheet = "LeftRightFeatures"))


rownames(TADPOLE_D1_D2_Dict) <- TADPOLE_D1_D2_Dict$FLDNAME

1.3 Conditioning the data

# mm3 to mm
isVolume <- c("Ventricles","Hippocampus","WholeBrain","Entorhinal","Fusiform","MidTemp","ICV",
              TADPOLE_D1_D2_Dict$FLDNAME[str_detect(TADPOLE_D1_D2_Dict$TEXT,"Volume")]
              )


#TADPOLE_D1_D2[,isVolume] <- apply(TADPOLE_D1_D2[,isVolume],2,'^',(1/3))
TADPOLE_D1_D2[,isVolume] <- TADPOLE_D1_D2[,isVolume]^(1/3)

# mm2 to mm
isArea <- TADPOLE_D1_D2_Dict$FLDNAME[str_detect(TADPOLE_D1_D2_Dict$TEXT,"Area")]
TADPOLE_D1_D2[,isArea] <- sqrt(TADPOLE_D1_D2[,isArea])

# Get only cross sectional measurements
FreeSurfersetCross <- str_detect(colnames(TADPOLE_D1_D2),"UCSFFSX")

# The subset of baseline measurements
baselineTadpole <- subset(TADPOLE_D1_D2,VISCODE=="bl")
table(baselineTadpole$DX)

                   Dementia Dementia to MCI             MCI MCI to Dementia 
          7             336               1             864               5 
  MCI to NL              NL       NL to MCI 
          2             521               1 

table(baselineTadpole$DX_bl)

AD CN EMCI LMCI SMC 342 417 310 562 106

rownames(baselineTadpole) <- baselineTadpole$PTID


validBaselineTadpole <- cbind(DX=baselineTadpole$DX_bl,
                                 AGE=baselineTadpole$AGE,
                                 Gender=1*(baselineTadpole$PTGENDER=="Female"),
                                 ADAS11=baselineTadpole$ADAS11,
                                 ADAS13=baselineTadpole$ADAS13,
                                 MMSE=baselineTadpole$MMSE,
                                 RAVLT_immediate=baselineTadpole$RAVLT_immediate,
                                 RAVLT_learning=baselineTadpole$RAVLT_learning,
                                 RAVLT_forgetting=baselineTadpole$RAVLT_forgetting,
                                 RAVLT_perc_forgetting=baselineTadpole$RAVLT_perc_forgetting,
                                 FAQ=baselineTadpole$FAQ,
                                 Ventricles=baselineTadpole$Ventricles,
                                 Hippocampus=baselineTadpole$Hippocampus,
                                 WholeBrain=baselineTadpole$WholeBrain,
                                 Entorhinal=baselineTadpole$Entorhinal,
                                 Fusiform=baselineTadpole$Fusiform,
                                 MidTemp=baselineTadpole$MidTemp,
                                 ICV=baselineTadpole$ICV,
                                 baselineTadpole[,FreeSurfersetCross])


LeftFields <- TADPOLE_D1_D2_Dict_LR$LFN
names(LeftFields) <- LeftFields
LeftFields <- LeftFields[LeftFields %in% colnames(validBaselineTadpole)]
RightFields <- TADPOLE_D1_D2_Dict_LR$RFN
names(RightFields) <- RightFields
RightFields <- RightFields[RightFields %in% colnames(validBaselineTadpole)]

## Normalize to ICV
validBaselineTadpole$Ventricles=validBaselineTadpole$Ventricles/validBaselineTadpole$ICV
validBaselineTadpole$Hippocampus=validBaselineTadpole$Hippocampus/validBaselineTadpole$ICV
validBaselineTadpole$WholeBrain=validBaselineTadpole$WholeBrain/validBaselineTadpole$ICV
validBaselineTadpole$Entorhinal=validBaselineTadpole$Entorhinal/validBaselineTadpole$ICV
validBaselineTadpole$Fusiform=validBaselineTadpole$Fusiform/validBaselineTadpole$ICV
validBaselineTadpole$MidTemp=validBaselineTadpole$MidTemp/validBaselineTadpole$ICV

leftData <- validBaselineTadpole[,LeftFields]/validBaselineTadpole$ICV
RightData <- validBaselineTadpole[,RightFields]/validBaselineTadpole$ICV

## get mean and relative difference 
meanLeftRight <- (leftData + RightData)/2
difLeftRight <- abs(leftData - RightData)
reldifLeftRight <- difLeftRight/meanLeftRight
colnames(meanLeftRight) <- paste("M",colnames(meanLeftRight),sep="_")
colnames(difLeftRight) <- paste("D",colnames(difLeftRight),sep="_")
colnames(reldifLeftRight) <- paste("RD",colnames(reldifLeftRight),sep="_")


validBaselineTadpole <- validBaselineTadpole[,!(colnames(validBaselineTadpole) %in% 
                                               c(LeftFields,RightFields))]
#validBaselineTadpole <- cbind(validBaselineTadpole,meanLeftRight,difLeftRight,reldifLeftRight)
validBaselineTadpole <- cbind(validBaselineTadpole,meanLeftRight,difLeftRight)

## Remove columns with too many NA more than %15 of NA
nacount <- apply(is.na(validBaselineTadpole),2,sum)/nrow(validBaselineTadpole) < 0.15
diagnose <- validBaselineTadpole$DX
pander::pander(table(diagnose))

AD	CN	EMCI	LMCI	SMC
342	417	310	562	106

validBaselineTadpole <- validBaselineTadpole[,nacount]
## Remove character columns
ischar <- sapply(validBaselineTadpole,class) == "character"
validBaselineTadpole <- validBaselineTadpole[,!ischar]
## Place back diagnose
validBaselineTadpole$DX <- diagnose


validBaselineTadpole <- validBaselineTadpole[complete.cases(validBaselineTadpole),]
ischar <- sapply(validBaselineTadpole,class) == "character"
validBaselineTadpole[,!ischar] <- sapply(validBaselineTadpole[,!ischar],as.numeric)

colnames(validBaselineTadpole) <- str_remove_all(colnames(validBaselineTadpole),"_UCSFFSX_11_02_15_UCSFFSX51_08_01_16")
colnames(validBaselineTadpole) <- str_replace_all(colnames(validBaselineTadpole)," ","_")
validBaselineTadpole$LONISID <- NULL
validBaselineTadpole$IMAGEUID <- NULL
validBaselineTadpole$LONIUID <- NULL

diagnose <- as.character(validBaselineTadpole$DX)
validBaselineTadpole$DX <- diagnose
pander::pander(table(validBaselineTadpole$DX))

AD	CN	EMCI	LMCI	SMC
245	359	272	444	93

validBaselineTadpole[validBaselineTadpole$DX %in% c("EMCI","LMCI"),"DX"] <- "MCI" 
validBaselineTadpole[validBaselineTadpole$DX %in% c("CN","SMC"),"DX"] <- "NL" 

pander::pander(table(validBaselineTadpole$DX))

AD	MCI	NL
245	716	452

1.4 Get the Time To Event on MCI Subjects

subjectsID <- rownames(validBaselineTadpole)
visitsID <- unique(TADPOLE_D1_D2$VISCODE)
baseDx <- TADPOLE_D1_D2[TADPOLE_D1_D2$VISCODE=="bl",c("PTID","DX","EXAMDATE")]
rownames(baseDx) <- baseDx$PTID 
baseDx <- baseDx[subjectsID,]
lastDx <- baseDx
toDementia <- baseDx
table(lastDx$DX)

   Dementia Dementia to MCI             MCI MCI to Dementia       MCI to NL 
        244               1             711               2               2 
         NL       NL to MCI 
        452               1 

hasDementia <- lastDx$PTID[str_detect(lastDx$DX,"Dementia")]


for (vid in visitsID)
{
  DxValue <- TADPOLE_D1_D2[TADPOLE_D1_D2$VISCODE==vid,c("PTID","DX","EXAMDATE")]
  rownames(DxValue) <- DxValue$PTID 
  DxValue <- DxValue[DxValue$PTID %in% subjectsID,]
  noDX <- DxValue$PTID[nchar(DxValue$DX) < 1]
  print(length(noDX))
  DxValue[noDX,] <- lastDx[noDX,]
  inLast <- lastDx$PTID[lastDx$PTID %in% DxValue$PTID]
  print(length(inLast))
  lastDx[inLast,] <- DxValue[inLast,]
  noDementia <- !(toDementia$PTID %in% hasDementia)
  toDementia[noDementia,] <- lastDx[noDementia,]
  hasDementia <- unique(c(hasDementia,lastDx$PTID[str_detect(lastDx$DX,"Dementia")]))
}

[1] 0 [1] 1413 [1] 2 [1] 1326 [1] 6 [1] 1218 [1] 23 [1] 1095 [1] 805 [1] 1058 [1] 29 [1] 710 [1] 20 [1] 212 [1] 14 [1] 167 [1] 32 [1] 553 [1] 25 [1] 298 [1] 18 [1] 130 [1] 667 [1] 667 [1] 112 [1] 112 [1] 176 [1] 176 [1] 177 [1] 177 [1] 625 [1] 625 [1] 251 [1] 251 [1] 159 [1] 159 [1] 7 [1] 7 [1] 17 [1] 99 [1] 9 [1] 63 [1] 1 [1] 1

table(lastDx$DX)

   Dementia Dementia to MCI             MCI MCI to Dementia       MCI to NL 
        428               2             463              80               7 
         NL  NL to Dementia       NL to MCI 
        406               1              26 

baseMCI <-baseDx$PTID[baseDx$DX == "MCI"]
lastDementia <- lastDx$PTID[str_detect(lastDx$DX,"Dementia")]
lastDementia2 <- toDementia$PTID[str_detect(toDementia$DX,"Dementia")]
lastNL <- lastDx$PTID[str_detect(lastDx$DX,"NL")]

MCIatBaseline <- baseDx[baseMCI,]
MCIatEvent <- toDementia[baseMCI,]
MCIatLast <- lastDx[baseMCI,]

MCIconverters <- MCIatBaseline[baseMCI %in% lastDementia,]
MCI_No_converters <- MCIatBaseline[!(baseMCI %in% MCIconverters$PTID),]
MCIconverters$TimeToEvent <- (as.Date(toDementia[MCIconverters$PTID,"EXAMDATE"]) 
                                   - as.Date(MCIconverters$EXAMDATE))

sum(MCIconverters$TimeToEvent ==0)

[1] 0

MCIconverters$AtEventDX <- MCIatEvent[MCIconverters$PTID,"DX"]
MCIconverters$LastDX <- MCIatLast[MCIconverters$PTID,"DX"]

MCI_No_converters$TimeToEvent <- (as.Date(lastDx[MCI_No_converters$PTID,"EXAMDATE"]) 
                                   - as.Date(MCI_No_converters$EXAMDATE))

MCI_No_converters$LastDX <- MCIatLast[MCI_No_converters$PTID,"DX"]

MCI_No_converters <- subset(MCI_No_converters,TimeToEvent > 0)

2 Prognosis MCI to AD Conversion

2.1 the set

MCIPrognosisIDs <- c(MCIconverters$PTID,MCI_No_converters$PTID)

TADPOLECrossMRI <- validBaselineTadpole[MCIPrognosisIDs,]
table(TADPOLECrossMRI$DX)

MCI 680

TADPOLECrossMRI$DX <- NULL
TADPOLECrossMRI$status <- 1*(rownames(TADPOLECrossMRI) %in% MCIconverters$PTID)
table(TADPOLECrossMRI$status)

0 1 436 244

2.1.0.1 Standarize the names for the reporting

studyName <- "TADPOLE"
dataframe <- TADPOLECrossMRI
outcome <- "status"

TopVariables <- 10

thro <- 0.60
cexheat = 0.15

2.2 Generaring the report

2.2.1 Libraries

Some libraries

library(psych)
library(whitening)
library("vioplot")
library("rpart")

2.2.2 Data specs

pander::pander(c(rows=nrow(dataframe),col=ncol(dataframe)-1))

rows	col
680	327

pander::pander(table(dataframe[,outcome]))

0	1
436	244

varlist <- colnames(dataframe)
varlist <- varlist[varlist != outcome]

largeSet <- length(varlist) > 1500 

2.2.3 Scaling the data

Scaling and removing near zero variance columns and highly co-linear(r>0.99999) columns

  ### Some global cleaning
  sdiszero <- apply(dataframe,2,sd) > 1.0e-16
  dataframe <- dataframe[,sdiszero]

  varlist <- colnames(dataframe)[colnames(dataframe) != outcome]
  tokeep <- c(as.character(correlated_Remove(dataframe,varlist,thr=0.99999)),outcome)
  dataframe <- dataframe[,tokeep]

  varlist <- colnames(dataframe)
  varlist <- varlist[varlist != outcome]
  
  iscontinous <- sapply(apply(dataframe,2,unique),length) >= 5 ## Only variables with enough samples



dataframeScaled <- FRESAScale(dataframe,method="OrderLogit")$scaledData

2.3 The heatmap of the data

numsub <- nrow(dataframe)
if (numsub > 1000) numsub <- 1000


if (!largeSet)
{

  hm <- heatMaps(data=dataframeScaled[1:numsub,],
                 Outcome=outcome,
                 Scale=TRUE,
                 hCluster = "row",
                 xlab="Feature",
                 ylab="Sample",
                 srtCol=45,
                 srtRow=45,
                 cexCol=cexheat,
                 cexRow=cexheat
                 )
  par(op)
}

2.3.0.1 Correlation Matrix of the Data

The heat map of the data

if (!largeSet)
{

  par(cex=0.6,cex.main=0.85,cex.axis=0.7)
  #cormat <- Rfast::cora(as.matrix(dataframe[,varlist]),large=TRUE)
  cormat <- cor(dataframe[,varlist],method="pearson")
  cormat[is.na(cormat)] <- 0
  gplots::heatmap.2(abs(cormat),
                    trace = "none",
  #                  scale = "row",
                    mar = c(5,5),
                    col=rev(heat.colors(5)),
                    main = "Original Correlation",
                    cexRow = cexheat,
                    cexCol = cexheat,
                     srtCol=45,
                     srtRow=45,
                    key.title=NA,
                    key.xlab="|Pearson Correlation|",
                    xlab="Feature", ylab="Feature")
  diag(cormat) <- 0
  print(max(abs(cormat)))
}

[1] 0.9996707

2.4 The decorrelation

DEdataframe <- IDeA(dataframe,verbose=TRUE,thr=thro)
#> 
#>  Included: 326 , Uni p: 0.008991707 , Uncorrelated Base: 152 , Outcome-Driven Size: 0 , Base Size: 152 
#> 
#> 
 1 <R=1.000,thr=0.900,N=   12>, Top: 6( 1 )[ 1 : 6 Fa= 6 : 0.900 ]( 6 , 6 , 0 ),<|>Tot Used: 12 , Added: 6 , Zero Std: 0 , Max Cor: 0.894
#> 
 2 <R=0.894,thr=0.750,N=   97>, Top: 36( 14 )[ 1 : 36 Fa= 38 : 0.750 ]( 35 , 51 , 6 ),<|>Tot Used: 94 , Added: 51 , Zero Std: 0 , Max Cor: 0.963
#> 
 3 <R=0.963,thr=0.900,N=   16>, Top: 8( 1 )[ 1 : 8 Fa= 45 : 0.900 ]( 8 , 8 , 38 ),<|>Tot Used: 101 , Added: 8 , Zero Std: 0 , Max Cor: 0.872
#> 
 4 <R=0.872,thr=0.750,N=   11>, Top: 5( 1 )[ 1 : 5 Fa= 48 : 0.750 ]( 4 , 6 , 45 ),<|>Tot Used: 106 , Added: 6 , Zero Std: 0 , Max Cor: 0.902
#> 
 5 <R=0.902,thr=0.900,N=    2>, Top: 1( 1 )[ 1 : 1 Fa= 49 : 0.900 ]( 1 , 1 , 48 ),<|>Tot Used: 106 , Added: 1 , Zero Std: 0 , Max Cor: 0.834
#> 
 6 <R=0.834,thr=0.750,N=    2>, Top: 1( 1 )[ 1 : 1 Fa= 50 : 0.750 ]( 1 , 1 , 49 ),<|>Tot Used: 106 , Added: 1 , Zero Std: 0 , Max Cor: 0.749
#> 
 7 <R=0.749,thr=0.600,N=  122>, Top: 31( 19 )[ 1 : 31 Fa= 73 : 0.600 ]( 30 , 57 , 50 ),<|>Tot Used: 163 , Added: 57 , Zero Std: 0 , Max Cor: 0.962
#> 
 8 <R=0.962,thr=0.900,N=    2>, Top: 1( 1 )[ 1 : 1 Fa= 74 : 0.900 ]( 1 , 1 , 73 ),<|>Tot Used: 164 , Added: 1 , Zero Std: 0 , Max Cor: 0.871
#> 
 9 <R=0.871,thr=0.750,N=   12>, Top: 6( 1 )[ 1 : 6 Fa= 78 : 0.750 ]( 6 , 6 , 74 ),<|>Tot Used: 168 , Added: 6 , Zero Std: 0 , Max Cor: 0.872
#> 
 10 <R=0.872,thr=0.750,N=   12>, Top: 3( 1 )[ 1 : 3 Fa= 79 : 0.750 ]( 3 , 3 , 78 ),<|>Tot Used: 168 , Added: 3 , Zero Std: 0 , Max Cor: 0.746
#> 
 11 <R=0.746,thr=0.600,N=   47>, Top: 13( 2 )[ 1 : 13 Fa= 88 : 0.600 ]( 13 , 26 , 79 ),<|>Tot Used: 188 , Added: 26 , Zero Std: 0 , Max Cor: 0.928
#> 
 12 <R=0.928,thr=0.900,N=    2>, Top: 1( 1 )[ 1 : 1 Fa= 89 : 0.900 ]( 1 , 1 , 88 ),<|>Tot Used: 189 , Added: 1 , Zero Std: 0 , Max Cor: 0.866
#> 
 13 <R=0.866,thr=0.750,N=    4>, Top: 2( 1 )[ 1 : 2 Fa= 91 : 0.750 ]( 2 , 2 , 89 ),<|>Tot Used: 189 , Added: 2 , Zero Std: 0 , Max Cor: 0.747
#> 
 14 <R=0.747,thr=0.600,N=   20>, Top: 9( 1 )[ 1 : 9 Fa= 93 : 0.600 ]( 9 , 9 , 91 ),<|>Tot Used: 191 , Added: 9 , Zero Std: 0 , Max Cor: 0.926
#> 
 15 <R=0.926,thr=0.900,N=    2>, Top: 1( 1 )[ 1 : 1 Fa= 94 : 0.900 ]( 1 , 1 , 93 ),<|>Tot Used: 191 , Added: 1 , Zero Std: 0 , Max Cor: 0.817
#> 
 16 <R=0.817,thr=0.750,N=    2>, Top: 1( 1 )[ 1 : 1 Fa= 95 : 0.750 ]( 1 , 1 , 94 ),<|>Tot Used: 191 , Added: 1 , Zero Std: 0 , Max Cor: 0.707
#> 
 17 <R=0.707,thr=0.600,N=    2>, Top: 1( 1 )[ 1 : 1 Fa= 96 : 0.600 ]( 1 , 1 , 95 ),<|>Tot Used: 191 , Added: 1 , Zero Std: 0 , Max Cor: 0.690
#> 
 18 <R=0.690,thr=0.600,N=    2>, Top: 1( 1 )[ 1 : 1 Fa= 96 : 0.600 ]( 1 , 1 , 96 ),<|>Tot Used: 191 , Added: 1 , Zero Std: 0 , Max Cor: 0.667
#> 
 19 <R=0.667,thr=0.600,N=    2>, Top: 1( 1 )[ 1 : 1 Fa= 97 : 0.600 ]( 1 , 1 , 96 ),<|>Tot Used: 191 , Added: 1 , Zero Std: 0 , Max Cor: 0.599
#> 
 20 <R=0.599,thr=0.600,N=    0>
#> 
 [ 20 ], 0.5992682 Decor Dimension: 191 Nused: 191 . Cor to Base: 116 , ABase: 45 , Outcome Base: 0 
#> 
varlistc <- colnames(DEdataframe)[colnames(DEdataframe) != outcome]

pander::pander(sum(apply(dataframe[,varlist],2,var)))

1377

pander::pander(sum(apply(DEdataframe[,varlistc],2,var)))

416

pander::pander(entropy(discretize(unlist(dataframe[,varlist]), 256)))

1.01

pander::pander(entropy(discretize(unlist(DEdataframe[,varlistc]), 256)))

0.762

2.4.1 The decorrelation matrix

if (!largeSet)
{

  par(cex=0.6,cex.main=0.85,cex.axis=0.7)
  
  UPSTM <- attr(DEdataframe,"UPSTM")
  
  gplots::heatmap.2(1.0*(abs(UPSTM)>0),
                    trace = "none",
                    mar = c(5,5),
                    col=rev(heat.colors(5)),
                    main = "Decorrelation matrix",
                    cexRow = cexheat,
                    cexCol = cexheat,
                   srtCol=45,
                   srtRow=45,
                    key.title=NA,
                    key.xlab="|Beta|>0",
                    xlab="Output Feature", ylab="Input Feature")
  
  par(op)
}

2.5 The heatmap of the decorrelated data

if (!largeSet)
{

  hm <- heatMaps(data=DEdataframe[1:numsub,],
                 Outcome=outcome,
                 Scale=TRUE,
                 hCluster = "row",
                 cexRow = cexheat,
                 cexCol = cexheat,
                 srtCol=45,
                 srtRow=45,
                 xlab="Feature",
                 ylab="Sample")
  par(op)
}

2.6 The correlation matrix after decorrelation

if (!largeSet)
{

  cormat <- cor(DEdataframe[,varlistc],method="pearson")
  cormat[is.na(cormat)] <- 0
  
  gplots::heatmap.2(abs(cormat),
                    trace = "none",
                    mar = c(5,5),
                    col=rev(heat.colors(5)),
                    main = "Correlation after IDeA",
                    cexRow = cexheat,
                    cexCol = cexheat,
                     srtCol=45,
                     srtRow=45,
                    key.title=NA,
                    key.xlab="|Pearson Correlation|",
                    xlab="Feature", ylab="Feature")
  
  par(op)
  diag(cormat) <- 0
  print(max(abs(cormat)))
}

[1] 0.5992682

2.7 U-MAP Visualization of features

2.7.1 The UMAP based on LASSO on Raw Data

if (nrow(dataframe) < 1000)
{
  classes <- unique(dataframe[1:numsub,outcome])
  raincolors <- rainbow(length(classes))
  names(raincolors) <- classes
  datasetframe.umap = umap(scale(dataframe[1:numsub,varlist]),n_components=2)
  plot(datasetframe.umap$layout,xlab="U1",ylab="U2",main="UMAP: Original",t='n')
  text(datasetframe.umap$layout,labels=dataframe[1:numsub,outcome],col=raincolors[dataframe[1:numsub,outcome]+1])
}

2.7.2 The decorralted UMAP

if (nrow(dataframe) < 1000)
{

  datasetframe.umap = umap(scale(DEdataframe[1:numsub,varlistc]),n_components=2)
  plot(datasetframe.umap$layout,xlab="U1",ylab="U2",main="UMAP: After IDeA",t='n')
  text(datasetframe.umap$layout,labels=DEdataframe[1:numsub,outcome],col=raincolors[DEdataframe[1:numsub,outcome]+1])
}

2.8 Univariate Analysis

2.8.1 Univariate

univarRAW <- uniRankVar(varlist,
               paste(outcome,"~1"),
               outcome,
               dataframe,
               rankingTest="AUC")

100 : M_ST24SA 200 : D_ST49TA 300 : D_ST47CV

univarDe <- uniRankVar(varlistc,
               paste(outcome,"~1"),
               outcome,
               DEdataframe,
               rankingTest="AUC",
               )

100 : M_ST24SA 200 : D_ST49TA 300 : La_D_ST47CV

2.8.2 Final Table

univariate_columns <- c("caseMean","caseStd","controlMean","controlStd","controlKSP","ROCAUC")

##top variables
topvar <- c(1:length(varlist)) <= TopVariables
tableRaw <- univarRAW$orderframe[topvar,univariate_columns]
pander::pander(tableRaw)

	caseMean	caseStd	controlMean	controlStd	controlKSP	ROCAUC
ADAS13	20.7611	6.16923	14.0091	5.78970	0.03549	0.788
ADAS11	12.8635	4.56128	8.7155	3.84978	0.00264	0.761
FAQ	5.4631	4.90262	1.9266	2.98257	0.00000	0.756
M_ST40CV	0.1799	0.00875	0.1875	0.00763	0.28199	0.750
M_ST29SV	0.1253	0.00708	0.1321	0.00750	0.58088	0.745
M_ST12SV	0.0913	0.00535	0.0962	0.00550	0.50030	0.744
Hippocampus	0.1582	0.00886	0.1664	0.00945	0.44340	0.737
RAVLT_immediate	29.0205	7.69236	37.2798	10.92838	0.04406	0.728
M_ST24CV	0.0996	0.00800	0.1059	0.00706	0.04673	0.727
M_ST31CV	0.1910	0.00945	0.1986	0.00902	0.94566	0.717

topLAvar <- univarDe$orderframe$Name[str_detect(univarDe$orderframe$Name,"La_")]
topLAvar <- unique(c(univarDe$orderframe$Name[topvar],topLAvar[1:as.integer(TopVariables/2)]))
finalTable <- univarDe$orderframe[topLAvar,univariate_columns]


pander::pander(finalTable)

	caseMean	caseStd	controlMean	controlStd	controlKSP	ROCAUC
ADAS11	12.86352	4.561284	8.71553	3.849783	2.64e-03	0.761
FAQ	5.46311	4.902619	1.92661	2.982574	0.00e+00	0.756
Hippocampus	0.15823	0.008856	0.16644	0.009452	4.43e-01	0.737
M_ST31TA	0.01874	0.001851	0.02006	0.001711	2.69e-01	0.703
M_ST36CV	0.15904	0.006107	0.16276	0.006117	5.31e-01	0.674
RAVLT_learning	3.17623	2.396347	4.66514	2.556538	1.35e-04	0.673
La_M_ST30SV	-0.00580	0.010722	-0.01268	0.011405	9.50e-01	0.669
M_ST39SA	0.34781	0.021709	0.36106	0.023122	4.59e-02	0.669
MMSE	26.91803	1.764255	27.96101	1.752723	2.78e-15	0.668
M_ST49CV	0.17209	0.007592	0.17643	0.008080	9.10e-01	0.651
La_M_ST11SV	0.04202	0.000658	0.04239	0.000695	8.29e-01	0.650
La_ADAS13	2.67226	2.070109	1.75325	1.988479	4.56e-01	0.631
La_M_ST37SV	0.00158	0.006665	0.00198	0.002163	3.04e-05	0.625
La_M_ST51CV	0.01531	0.001446	0.01589	0.001451	4.83e-01	0.614

dc <- getLatentCoefficients(DEdataframe)
fscores <- attr(DEdataframe,"fscore")


pander::pander(c(mean=mean(sapply(dc,length)),total=length(dc),fraction=length(dc)/(ncol(dataframe)-1)))

mean	total	fraction
2.49	126	0.385

theCharformulas <- attr(dc,"LatentCharFormulas")


finalTable <- rbind(finalTable,tableRaw[topvar[!(topvar %in% topLAvar)],univariate_columns])


orgnamez <- rownames(finalTable)
orgnamez <- str_remove_all(orgnamez,"La_")
finalTable$RAWAUC <- univarRAW$orderframe[orgnamez,"ROCAUC"]
finalTable$DecorFormula <- theCharformulas[rownames(finalTable)]
finalTable$fscores <- fscores[rownames(finalTable)]

Final_Columns <- c("DecorFormula","caseMean","caseStd","controlMean","controlStd","controlKSP","ROCAUC","RAWAUC","fscores")

finalTable <- finalTable[order(-finalTable$ROCAUC),]
pander::pander(finalTable[,Final_Columns])

	DecorFormula	caseMean	caseStd	controlMean	controlStd	controlKSP	ROCAUC	RAWAUC	fscores
ADAS13	NA	20.76107	6.169227	14.00911	5.79e+00	3.55e-02	0.788	0.788	NA
ADAS11	NA	12.86352	4.561284	8.71553	3.85e+00	2.64e-03	0.761	0.761	2
ADAS111	NA	12.86352	4.561284	8.71553	3.85e+00	2.64e-03	0.761	NA	NA
FAQ	NA	5.46311	4.902619	1.92661	2.98e+00	0.00e+00	0.756	0.756	NA
FAQ1	NA	5.46311	4.902619	1.92661	2.98e+00	0.00e+00	0.756	NA	NA
M_ST40CV	NA	0.17986	0.008748	0.18753	7.63e-03	2.82e-01	0.750	0.750	NA
M_ST29SV	NA	0.12530	0.007075	0.13210	7.50e-03	5.81e-01	0.745	0.745	NA
M_ST12SV	NA	0.09133	0.005353	0.09622	5.50e-03	5.00e-01	0.744	0.744	NA
Hippocampus	NA	0.15823	0.008856	0.16644	9.45e-03	4.43e-01	0.737	0.737	6
Hippocampus1	NA	0.15823	0.008856	0.16644	9.45e-03	4.43e-01	0.737	NA	NA
RAVLT_immediate	NA	29.02049	7.692361	37.27982	1.09e+01	4.41e-02	0.728	0.728	NA
M_ST24CV	NA	0.09963	0.008002	0.10594	7.06e-03	4.67e-02	0.727	0.727	NA
M_ST31CV	NA	0.19103	0.009453	0.19857	9.02e-03	9.46e-01	0.717	0.717	NA
M_ST31TA	NA	0.01874	0.001851	0.02006	1.71e-03	2.69e-01	0.703	0.703	39
M_ST36CV	NA	0.15904	0.006107	0.16276	6.12e-03	5.31e-01	0.674	0.674	5
RAVLT_learning	NA	3.17623	2.396347	4.66514	2.56e+00	1.35e-04	0.673	0.673	NA
La_M_ST30SV	- (0.319)Ventricles + M_ST30SV	-0.00580	0.010722	-0.01268	1.14e-02	9.50e-01	0.669	0.681	-1
M_ST39SA	NA	0.34781	0.021709	0.36106	2.31e-02	4.59e-02	0.669	0.669	NA
MMSE	NA	26.91803	1.764255	27.96101	1.75e+00	2.78e-15	0.668	0.668	NA
M_ST49CV	NA	0.17209	0.007592	0.17643	8.08e-03	9.10e-01	0.651	0.651	NA
La_M_ST11SV	+ (1.57e-04)ICV + M_ST11SV	0.04202	0.000658	0.04239	6.95e-04	8.29e-01	0.650	0.645	-1
La_ADAS13	- (1.406)ADAS11 + ADAS13	2.67226	2.070109	1.75325	1.99e+00	4.56e-01	0.631	0.788	-1
La_M_ST37SV	- (0.774)Ventricles + M_ST37SV	0.00158	0.006665	0.00198	2.16e-03	3.04e-05	0.625	0.601	-1
La_M_ST51CV	+ (2.345)M_ST31TA - (3.273)M_ST51TA + (0.144)M_ST31SA - (0.201)M_ST51SA - (0.645)M_ST31CV + M_ST51CV	0.01531	0.001446	0.01589	1.45e-03	4.83e-01	0.614	0.648	-4

2.9 Comparing IDeA vs PCA vs EFA

2.9.1 PCA

featuresnames <- colnames(dataframe)[colnames(dataframe) != outcome]
pc <- prcomp(dataframe[,iscontinous],center = TRUE,scale. = TRUE)   #principal components
predPCA <- predict(pc,dataframe[,iscontinous])
PCAdataframe <- as.data.frame(cbind(predPCA,dataframe[,!iscontinous]))
colnames(PCAdataframe) <- c(colnames(predPCA),colnames(dataframe)[!iscontinous]) 
#plot(PCAdataframe[,colnames(PCAdataframe)!=outcome],col=dataframe[,outcome],cex=0.65,cex.lab=0.5,cex.axis=0.75,cex.sub=0.5,cex.main=0.75)

#pander::pander(pc$rotation)


PCACor <- cor(PCAdataframe[,colnames(PCAdataframe) != outcome])


  gplots::heatmap.2(abs(PCACor),
                    trace = "none",
  #                  scale = "row",
                    mar = c(5,5),
                    col=rev(heat.colors(5)),
                    main = "PCA Correlation",
                    cexRow = 0.5,
                    cexCol = 0.5,
                     srtCol=45,
                     srtRow= -45,
                    key.title=NA,
                    key.xlab="Pearson Correlation",
                    xlab="Feature", ylab="Feature")

2.9.2 EFA

EFAdataframe <- dataframeScaled

if (length(iscontinous) < 2000)
{
  topred <- min(length(iscontinous),nrow(dataframeScaled),ncol(predPCA)/2)
  if (topred < 2) topred <- 2
  
  uls <- fa(dataframeScaled[,iscontinous],nfactors=topred,rotate="varimax",warnings=FALSE)  # EFA analysis
  predEFA <- predict(uls,dataframeScaled[,iscontinous])
  EFAdataframe <- as.data.frame(cbind(predEFA,dataframeScaled[,!iscontinous]))
  colnames(EFAdataframe) <- c(colnames(predEFA),colnames(dataframeScaled)[!iscontinous]) 


  
  EFACor <- cor(EFAdataframe[,colnames(EFAdataframe) != outcome])
  
  
    gplots::heatmap.2(abs(EFACor),
                      trace = "none",
    #                  scale = "row",
                      mar = c(5,5),
                      col=rev(heat.colors(5)),
                      main = "EFA Correlation",
                      cexRow = 0.5,
                      cexCol = 0.5,
                       srtCol=45,
                       srtRow= -45,
                      key.title=NA,
                      key.xlab="Pearson Correlation",
                      xlab="Feature", ylab="Feature")
}

2.10 Effect on CAR modeling

par(op)
par(xpd = TRUE)
dataframe[,outcome] <- factor(dataframe[,outcome])
rawmodel <- rpart(paste(outcome,"~."),dataframe,control=rpart.control(maxdepth=3))
pr <- predict(rawmodel,dataframe,type = "class")

  ptab <- list(er="Error",detail=matrix(nrow=6,ncol=1))
  if (length(unique(pr))>1)
  {
    plot(rawmodel,main="Raw",branch=0.5,uniform = TRUE,compress = TRUE,margin=0.1)
    text(rawmodel, use.n = TRUE,cex=0.75)
    ptab <- epiR::epi.tests(table(pr==0,dataframe[,outcome]==0))
  }

pander::pander(table(dataframe[,outcome],pr))

	0	1
0	330	106
1	36	208

pander::pander(ptab$detail[c(5,3,4,6),])

	statistic	est	lower	upper
5	diag.ac	0.791	0.759	0.821
3	se	0.852	0.802	0.894
4	sp	0.757	0.714	0.796
6	diag.or	17.987	11.866	27.267

par(op)
par(xpd = TRUE)
DEdataframe[,outcome] <- factor(DEdataframe[,outcome])
IDeAmodel <- rpart(paste(outcome,"~."),DEdataframe,control=rpart.control(maxdepth=3))
pr <- predict(IDeAmodel,DEdataframe,type = "class")

  ptab <- list(er="Error",detail=matrix(nrow=6,ncol=1))
  if (length(unique(pr))>1)
  {
    plot(IDeAmodel,main="IDeA",branch=0.5,uniform = TRUE,compress = TRUE,margin=0.1)
    text(IDeAmodel, use.n = TRUE,cex=0.75)
    ptab <- epiR::epi.tests(table(pr==0,DEdataframe[,outcome]==0))
  }

pander::pander(table(DEdataframe[,outcome],pr))

	0	1
0	371	65
1	89	155

pander::pander(ptab$detail[c(5,3,4,6),])

	statistic	est	lower	upper
5	diag.ac	0.774	0.740	0.804
3	se	0.635	0.571	0.696
4	sp	0.851	0.814	0.883
6	diag.or	9.940	6.862	14.401

par(op)
par(xpd = TRUE)
PCAdataframe[,outcome] <- factor(PCAdataframe[,outcome])
PCAmodel <- rpart(paste(outcome,"~."),PCAdataframe,control=rpart.control(maxdepth=3))
pr <- predict(PCAmodel,PCAdataframe,type = "class")
ptab <- list(er="Error",detail=matrix(nrow=6,ncol=1))
if (length(unique(pr))>1)
{
  plot(PCAmodel,main="PCA",branch=0.5,uniform = TRUE,compress = TRUE,margin=0.1)
  text(PCAmodel, use.n = TRUE,cex=0.75)
  ptab <- epiR::epi.tests(table(pr==0,PCAdataframe[,outcome]==0))
}

pander::pander(table(PCAdataframe[,outcome],pr))

	0	1
0	401	35
1	122	122

pander::pander(ptab$detail[c(5,3,4,6),])

	statistic	est	lower	upper
5	diag.ac	0.769	0.736	0.800
3	se	0.500	0.436	0.564
4	sp	0.920	0.890	0.943
6	diag.or	11.457	7.476	17.559

par(op)

2.10.1 EFA

  EFAdataframe[,outcome] <- factor(EFAdataframe[,outcome])
  EFAmodel <- rpart(paste(outcome,"~."),EFAdataframe,control=rpart.control(maxdepth=3))
  pr <- predict(EFAmodel,EFAdataframe,type = "class")
  
  ptab <- list(er="Error",detail=matrix(nrow=6,ncol=1))
  if (length(unique(pr))>1)
  {
    plot(EFAmodel,main="EFA",branch=0.5,uniform = TRUE,compress = TRUE,margin=0.1)
    text(EFAmodel, use.n = TRUE,cex=0.75)
    ptab <- epiR::epi.tests(table(pr==0,EFAdataframe[,outcome]==0))
  }

  pander::pander(table(EFAdataframe[,outcome],pr))

	0	1
0	383	53
1	91	153

  pander::pander(ptab$detail[c(5,3,4,6),])

	statistic	est	lower	upper
5	diag.ac	0.788	0.756	0.818
3	se	0.627	0.563	0.688
4	sp	0.878	0.844	0.908
6	diag.or	12.150	8.250	17.893

  par(op)